# Project URL: https://code.google.com/p/unihan-52plus-sqlite-3-db/
#
# Original project url: http://code.google.com/p/unihan-sqlite-3-database/
# Original Author: david2ansands@gmail.com
#
# Modified by https://code.google.com/p/unihan-52plus-sqlite-3-db/ 
# for *nix compatibility and some improvement
#
#

function usage(){
    usage_str = \
	"awk -f [name_of_this_awk] [name_of_merged_unihan_txt]" "\n" \
	"e.g. awk -f uni_db_unix.awk Unihan_merged.txt"
}

BEGIN{
  # delete any .dx or .idx files -- they will contain tab separated import data
  system("rm -f *.*dx")

  print "Constructing a series of data files from Unihan.txt. Please be patient ..."
  maintable = "kBase"
}

# main loop for every line in Unihan.txt 

 /^#/ { print $0 >  "uni_header.txt"; next }

 { 
    # remove the "U+" prefix on datapoints, which begin each line 
   sub(/^../,"",$0)
 
    # avoid special characters in data field by taking a substring
    data =  substr($0,length($1 $2)+3)

    if ($2 == "kDefinition")
       # write datapoint, utf-8 char and definition to main file
       print $1 "\t" ch_utf8(strtonum("0x" $1)) "\t" data > maintable".idx"
    else {
       # write import file to "TagName.dx"
       print $1 "\t" data > $2".dx"
       close($2."dx")
    }
 }


END{
  # the name of the SQLite command file which will create the database
  qryfile = "unihan.qry"
  
  # the Unihan database name
  dbname = "unihan.sl3"

  print "Creating sqlite database: '" dbname "' from generated data files..."
  
  #delete any database with our target name 
  system ("rm -f " dbname)
  
  # create the commands for SQLite
  print "PRAGMA encoding = 'UTF-8';" > qryfile
  print "create table " maintable " (k text primary key, char text, def text);" > qryfile
  print "create index k on " maintable "(k);" > qryfile
  print ".mode tabs" > qryfile
  print ".import " maintable".idx " maintable " ;" > qryfile
  cmd_qry = "sqlite3 " dbname " < " qryfile
  system(cmd_qry)
  close(cmd_qry)

  
  # create the rest of the tables
  
  # get a diretory of the recently created files
  cmd_ls = "find . -iname \"*.dx\" -printf \"%f \""
  cmd_ls | getline lst_dx_str
  close(cmd_ls)
  split(lst_dx_str,lst_dx," ")
  for( dx in lst_dx ){
      fn = lst_dx[dx]
      print "looking at file " fn
      # check to see if it is a data import file and if so build a table and import data
      if (substr(fn,1,1) == "k"){
	  importfile = fn
	  tablename = substr(importfile,1,length(importfile)-3)
	  print "create table " tablename " (k text primary key, v text);" > qryfile
	  # we must tell sqlite3 that the fileds are separated by tabular char
	  print ".separator \"\\t\"" > qryfile
	  # import data from dx file
	  print ".import " importfile " " tablename > qryfile
	  # close the file handle to flush the file
	  close(qryfile)
	  # execute the commands in the file, "unihan.qry"
	  cmd_qry = "sqlite3 " dbname " < " qryfile
	  system(cmd_qry)
	  close(cmd_qry)

      }
    }
  
  print "Unihan Sqlite3 database reation successfully done."
  print "Removing intermediate files."
  system("rm -f *.*dx unihan.qry")
}
#------------------------ functions for printing utf characters ------------------------

func ch_utf8(dp){
if (dp > 0xFFFF) 
    return ch_utf8_6(dp)
   u1 = or(rshift(dp,12),0xE0)
   u2 = or(rshift(and(dp,0x0FC0),6),0x80)
   u3 = or(and(dp,0x003F),0x80)
   return sprintf("%c%c%c",u1,u2,u3)
}  

func ch_utf8_6(dp){
   u1 = or(rshift(and(dp,0x070000),19),0xF0)
   u2 = or(rshift(and(dp,0x03F000),12),0x80)
   u3 = or(rshift(and(dp,0x000FC0),6),0x80)
   u4 = or(and(dp,0x00003F),0x80)
   return sprintf("%c%c%c%c",u1,u2,u3,u4)
}
